{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "import torch\n",
    "import numpy as np\n",
    "from torch import nn, optim\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm_notebook\n",
    "import matplotlib.pyplot as plt\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torchvision.utils import save_image\n",
    "\n",
    "from dataset import TextDataset                          # custom module\n",
    "from model.seq2seq import AttnDecoderRNN, DecoderRNN, EncoderRNN\n",
    "\n",
    "\n",
    "use_cuda = torch.cuda.is_available()                     # gpu可用\n",
    "# use_cuda = False\n",
    "device = torch.device('cuda' if use_cuda else 'cpu')     # 优先使用gpu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.读入数据，设置参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SOS_token = 0\n",
    "EOS_token = 1\n",
    "MAX_LENGTH = 10\n",
    "SOS_token = 0                                           # 词汇表中的起止符\n",
    "EOS_token = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lines...\n",
      "Read 135842 sentence pairs\n",
      "Trimmed to 10853 sentence pairs\n",
      "Counting words...\n",
      "Counted words:\n",
      "fra 4489\n",
      "eng 2925\n",
      "['elle fait des progres en chinois .', 'she is progressing in chinese .']\n"
     ]
    }
   ],
   "source": [
    "lang_dataset = TextDataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang_loader = DataLoader(lang_dataset, shuffle=True)   # use num_workers=4 can be slower???"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source language: tensor([[   9, 1454, 1167,   42,    5,    1]])\n",
      "target language: tensor([[  2,   3,  23, 691, 490,   4,   1]])\n",
      "Wall time: 243 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for data in lang_loader:\n",
    "    x, y = data\n",
    "    print('source language:', x)\n",
    "    print('target language:', y)\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 注意数据集批次数据末尾的1表示`<EOS>`结束符，每个数据对都有"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source language vocabulary size: 4489\n",
      "target language vocabulary size: 2925\n"
     ]
    }
   ],
   "source": [
    "input_size = lang_dataset.input_lang_words\n",
    "output_size = lang_dataset.output_lang_words\n",
    "print('source language vocabulary size:', input_size)\n",
    "print('target language vocabulary size:', output_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define hyperparameters\n",
    "hidden_size = 128\n",
    "epochs = 20\n",
    "batch_size = 1\n",
    "use_attn = False       # the sign whether to use attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the model\n",
    "encoder = EncoderRNN(vocab_size=input_size, hidden_size=hidden_size, n_layers=2)         # prefer using gpu\n",
    "decoder = DecoderRNN(vocab_size=output_size, hidden_size=hidden_size, n_layers=2)\n",
    "attn_decoder = AttnDecoderRNN(vocab_size=output_size, hidden_size=hidden_size, n_layers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------epoch:1---------------\n",
      "500/10853, Loss:21.422131\n",
      "1000/10853, Loss:20.050512\n",
      "1500/10853, Loss:19.707606\n",
      "2000/10853, Loss:18.458258\n",
      "2500/10853, Loss:17.537111\n",
      "3000/10853, Loss:17.721788\n",
      "3500/10853, Loss:17.243726\n",
      "4000/10853, Loss:16.622123\n",
      "4500/10853, Loss:17.600505\n",
      "5000/10853, Loss:16.238710\n",
      "5500/10853, Loss:15.749693\n",
      "6000/10853, Loss:15.687889\n",
      "6500/10853, Loss:15.487883\n",
      "7000/10853, Loss:15.547331\n",
      "7500/10853, Loss:14.931321\n",
      "8000/10853, Loss:15.572950\n",
      "8500/10853, Loss:15.032901\n",
      "9000/10853, Loss:15.266654\n",
      "9500/10853, Loss:15.104083\n",
      "10000/10853, Loss:15.196235\n",
      "10500/10853, Loss:13.543949\n",
      "Finish 1/1, Loss:16.569376, Time:656s\n"
     ]
    }
   ],
   "source": [
    "# train the translation model\n",
    "import time\n",
    "plot_losses = []\n",
    "def train(encoder, decoder, epochs, use_attn):\n",
    "    \"\"\"\n",
    "        func: train the seq2seq translation\n",
    "        encoder: RNN encoding model\n",
    "        decoder: RNN decoding model\n",
    "        epochs: the number of iteration\n",
    "        use_attn: whether to use attention, True represents use \n",
    "    \"\"\"\n",
    "    param = list(encoder.parameters()) + list(decoder.parameters())         # the parameters preparaed to optimize\n",
    "    optimizer = optim.Adam(param, lr=1e-3)                                  # define the opimizer\n",
    "    criterion = nn.NLLLoss()                                                # define the negative log likelihood loss\n",
    "    for epoch in range(epochs):\n",
    "        start = time.time()                                                 # calculate consumption time\n",
    "        running_loss = 0                                                    # loss value during training\n",
    "        total_loss = 0                                                      # total loss after training\n",
    "        plt_loss_total = 0                                                  # loss value for plot\n",
    "        print('{}epoch:{}{}'.format('-'*15, epoch+1, '-'*15))\n",
    "        for i, data in enumerate(lang_loader):\n",
    "            in_lang, out_lang = data\n",
    "#             print(in_lang.shape, out_lang.shape)\n",
    "            GPU = lambda x:x.to(device)                                     # prefer using gpu\n",
    "            in_lang, out_lang = map(GPU, [in_lang, out_lang])               \n",
    "            encoder, decoder = map(GPU, [encoder, decoder])\n",
    "            \n",
    "            # 1.encode source language to a context\n",
    "            # encoder_outputs is used to attention decoder，该变量只用于注意力解码中\n",
    "            encoder_outputs = torch.zeros([batch_size, MAX_LENGTH, hidden_size]).to(device)  # create a zero output\n",
    "            encoder_hidden  = encoder.initHidden().to(device)               # initialize h0 state\n",
    "            # in_lang:(N,seq)            \n",
    "            for seq_idx in range(in_lang.shape[1]):\n",
    "                encoder_output, encoder_hidden = encoder(in_lang[:, seq_idx:seq_idx+1], encoder_hidden)\n",
    "                encoder_outputs[:,seq_idx:seq_idx+1] = encoder_output[:,:]           # [1, 1, hidden]\n",
    "#                 print(encoder_outputs[:,seq_idx:seq_idx+1].shape,  encoder_output[:,:].shape)            \n",
    "#             print(encoder_outputs.shape)                                 # (1,10,hidden)\n",
    "            \n",
    "            # 2.decode context to seqence\n",
    "            decoder_input = torch.LongTensor([[SOS_token]]).to(device)     # note [[]] =>  get (1,1)\n",
    "            decoder_hidden = encoder_hidden                                # (1,1,hidden) context tensor !!!\n",
    "            loss = 0\n",
    "            # 2.1 use attention to decode context\n",
    "            if use_attn:\n",
    "                for seq_idx in range(out_lang.shape[1]):\n",
    "                    # decoder_output: (1,1,vocab_size) decoder_hidden: (1,1,hidden)\n",
    "                    # decoder_attention: (1, 1, MAX_LENGTH)\n",
    "                    decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden,\n",
    "                                                                                encoder_outputs)\n",
    "                    \n",
    "                    # predict: (1,1,vocab_size)  label:(1)\n",
    "                    loss += criterion(decoder_output.view(1,-1), out_lang[:,seq_idx:seq_idx+1][0])\n",
    "                    topv,topi = decoder_output.data.topk(1)\n",
    "                    if topi[0][0] == EOS_token:\n",
    "                        break\n",
    "                    decoder_input = out_lang[:,seq_idx:seq_idx+1]      # (1,1)\n",
    "            # 2.2 use no attention to decode context                    \n",
    "            else:      \n",
    "                for seq_idx in range(out_lang.shape[1]):\n",
    "                    # decoder_output: (1,1,vocab_size) decoder_hidden: (1,1,hidden)\n",
    "                    decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden)\n",
    "                                        \n",
    "                    # predict:(1,vocab_size)   label:(1)\n",
    "                    loss += criterion(decoder_output.view(1,-1), out_lang[:,seq_idx:seq_idx+1][0])\n",
    "                    topv, topi = decoder_output.data.topk(1)          # select the maximum value\n",
    "                    if topi.item() == EOS_token:                      # finish decoding\n",
    "                        break\n",
    "                    # teacher forcing指在训练过程中直接使用正确的标签来进行解码器的训练，当然可以取当前时刻输出的\n",
    "                    # 最大值作为下一个时刻的输入，但teacher forcing可以加速收敛，但在测试时下一个时刻的输入必须是当前\n",
    "                    # 时刻输出的值(输出字典大小的向量，取最大的值对应的索引当做输出的词)\n",
    "                    decoder_input = out_lang[:,seq_idx:seq_idx+1]   # use teacher forcing to accelerate convergence!!!\n",
    "                    \n",
    "            optimizer.zero_grad()           # clear the gradient\n",
    "            loss.backward()                 # backpropagation\n",
    "            optimizer.step()                # update the parameters\n",
    "            running_loss += loss.item()\n",
    "            plt_loss_total += loss.item()\n",
    "            total_loss += loss.item()\n",
    "            if (i + 1) % 500 == 0:\n",
    "                print('{}/{}, Loss:{:.6f}'.format(i+1, len(lang_loader), running_loss/500))\n",
    "                running_loss = 0\n",
    "            if (i + 1) % 100 == 0:\n",
    "                plot_loss = plt_loss_total / 100\n",
    "                plot_losses.append(plot_loss)\n",
    "                plt_loss_total = 0\n",
    "\n",
    "        epoch_time = time.time() - start                            # calculate consumption time\n",
    "        print('Finish {}/{}, Loss:{:.6f}, Time:{:.0f}s'.format(epoch+1, epochs, \n",
    "                                                               total_loss/len(lang_loader), epoch_time))\n",
    "    # save the model\n",
    "    attn = 'attn_' if use_attn else ''\n",
    "    torch.save(encoder.state_dict(), os.path.join('snapshot', 'encoder.pth'))\n",
    "    torch.save(decoder.state_dict(), os.path.join('snapshot', attn+'decoder.pth'))\n",
    "\n",
    "train(encoder, decoder, 1, use_attn=False)\n",
    "# train(encoder, attn_decoder, 2, use_attn=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1367aaf4940>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzsvXeYY2d59/+91bs0M5q2M7PN273N9q5xtzHBMQZjeEOHYGIS8hJIIJAQWvKDN3kTEiDg/EIKsQ0ktAQwxnQb2xj3svb2XmenV7VRl573j3Oeo6M2I82MRjPS/bmuvXZ0dCQ9Z7T7fM/dSQgBhmEYpnkx1HsBDMMwTH1hIWAYhmlyWAgYhmGaHBYChmGYJoeFgGEYpslhIWAYhmlyWAiYhoGIzhPRb9V7HQyz0mAhYBiGaXJYCBimjhCRqd5rYBgWAqYhISIrEX2ZiIbUP18mIqv6nJ+IfkJEASKaIqIniMigPvcXRDRIRGEiOkFEryrz/nYi+iIRXSCiIBE9qR67iYgGCs7VXFZE9Bki+j4RfZOIQgA+SUQxImrVnX8ZEU0QkVl9fBcRHSOiaSL6JRGtqdGvjWlSWAiYRuVTAK4CsBvALgBXAvi0+txHAQwAaAfQCeCTAAQRbQbwQQB7hRBuAL8N4HyZ9/8CgCsAXAOgFcDHAGQrXNsdAL4PwAfg8wCeAfA7uuffAeD7QogUEb1BXd//Utf7BIDvVPg5DFMRLARMo/JOAP9HCDEmhBgH8FkAv6s+lwLQDWCNECIlhHhCKE23MgCsALYRkVkIcV4IcabwjVXr4S4AHxJCDAohMkKIp4UQiQrX9owQ4gEhRFYIEQPwbQBvV9+bALxNPQYAfwjg74QQx4QQaQB/C2A3WwXMYsJCwDQqqwBc0D2+oB4DlLvw0wAeIqKzRPRxABBCnAbwYQCfATBGRN8lolUoxg/ABqBIJCrkYsHj7wO4Wv2sGwAIKHf+ALAGwN2qGysAYAoAAeiZ52czTBEsBEyjMgRlE5WsVo9BCBEWQnxUCLEewO0APiJjAUKIbwshrlNfKwD8fYn3ngAQB3BJiedmADjkAyIyQnHp6Mlr+SuECAB4CMBboLiFviNybYEvAvhDIYRP98cuhHh6zt8Aw1QICwHTqHwHwKeJqJ2I/AD+CsA3AYCIXkdEG1Q3TAiKSyhDRJuJ6GY1qBwHEFOfy0MIkQVwH4B/JKJVRGQkoqvV150EYCOi16rB3k9DcTfNxbcBvBtKrODbuuP/BuATRHSpunYvEb15Hr8PhikLCwHTqPwNgBcBHARwCMBL6jEA2AjgVwAiUAK1/yKE+DWUDftzUO74RwB0QAnUluLP1Pd9AYq75u8BGIQQQQB/BOAeAINQLISBMu+h50F1XaNCiAPyoBDih+p7f1fNMjoM4DUVvB/DVAzxYBqGYZjmhi0ChmGYJoeFgGEYpslhIWAYhmlyWAgYhmGanBXR8Mrv94u1a9fWexkMwzArin379k0IIQrrWIpYEUKwdu1avPjii/VeBsMwzIqCiC7MfRa7hhiGYZoeFgKGYZgmh4WAYRimyWEhYBiGaXJYCBiGYZocFgKGYZgmh4WAYRimyWkaIThwMYCX+6frvQyGYZhlR9MIwaceOIS/+emxei+DYRhm2dEUQpDKZHFyJIJgLFXvpTAMwyw7mkIIzoxHkMxkEY6zEDAMwxTSFEJwbDgEAAjH03VeCcMwzPKjKYTg6JAiBNFkBulMts6rYRiGWV40hRAcGw5rP0cSbBUwDMPoaXghEELg6HAIVpNyqeweYhiGyafhhWAsnMDUTBKXr24BAIQ4YMwwDJNHwwuBjA9cua4VAFsEDMMwhTS+EKgZQ69gIWAYhilJUwhBb4sdq3x2AOBaAoZhmAIaXgiODYewrdsDt00Zz8wWAcMwTD4NLQTRZBrnJmawtdsDt80MgC0ChmGYQhpaCE6MhCEEsG2VBxaTAVaTgS0ChmGYAhpaCGSgeFu3BwDgtpkRYiFgGIbJo2ZCQER9RPQYER0joiNE9CH1+OeJ6DgRHSSiHxKRr1ZrODYcgttqQm+LEij22EzsGmIYhimglhZBGsBHhRBbAVwF4ANEtA3AwwC2CyF2AjgJ4BO1WsB7rlmLL79tN4gIAOC2mdg1xDAMU4CpVm8shBgGMKz+HCaiYwB6hBAP6U57FsCbarWGDR1ubOhwa4/dNjNbBAzDMAUsSYyAiNYCuAzAcwVP3QXg52Ve8z4iepGIXhwfH1+UdbisbBEwDMMUUnMhICIXgB8A+LAQIqQ7/iko7qNvlXqdEOKrQog9Qog97e3ti7IWdg0xDMMUUzPXEAAQkRmKCHxLCHG/7vidAF4H4FVCCFHLNehh1xDDMEwxNRMCUiK09wI4JoT4R93xWwH8BYAbhRDRWn1+Kdw2E2aSGWSyAkYDLeVHMwzDLFtq6Rq6FsDvAriZiParf24D8M8A3AAeVo/9Ww3XkIdsMxFh9xDDMIxGLbOGngRQ6rb7Z7X6zLnwqG0mQvEUvA5zvZbBMAyzrGjoyuJCuPEcwzBMMU0mBNx4jmEYppAmEwK2CBiGYQppTiFIsEXAMAwjaTIhUFxDnDXEMAyTo8mEQLEIuBU1wzBMjqYSApvZCIuRh9MwDMPoaSohAGS/IY4RMAzDSJpUCNgiYBiGkTShEFTXeG4oEMMvj4zUcEUMwzD1pQmFoDqL4L4nz+F/f3MfZhJsRTAM05iwEMzBhakohADOjEdquCqGYZj60YRCUJ1r6OKU0in71CgLAcMwjUkTCkG+RbD/YgDBWGlhEEKgXwrBGAsBwzCNSRMKgRmRZBrZrMBgIIY3/stTeOu/P4OpmWTRuZMzSUSTGQDA6bHwUi+VYRhmSWg6IfDYTBACiCTTePLUOIQATo9F8I7/eLZIDKQ14Laa2CJgGKZhaToh0HcgfeLUBDrcVnzt9/bi3MQM3nnPc3nxAxkfuGFTO/qnooinMnVZM8MwTC1pQiFQGs8Foyk8dXoC12304/qN7fjKOy7HseEQHj46qp0rheCVWzo4c4hhmIalCYVAsQieOzeJ6WgK12/0A1A2e5vZgMODIe3c/qkoOtxW7Oz1AlBcSAzDMI1GEwqBYhH8/JBSLXztBkUIjAbCtm4PDg8FtXP7p6Loa3VgbZsTRgNxCinDMA1JEwqBYhG8cGEKW7rc6HDbtOe293hxdCiEbFYAAC5OxbC61QGLyYC1bQ6c4swhhmEakKYVAiGA61RrQLJ9lReRRBoXpqJIprMYCsbQ1+oAAGzscHPmEMMwDUnTCYFHdQ0BwPWb2vOeu7THAwA4MhTEUCAGIYDVUgg6XbgwGUUizZlDDMM0Fk0nBFaTAWYjwWI04Mq1rXnPbexww2JUAsayhkAKwYYOFzJZgXMTM0u+ZoZhmFrSdEJARHDbzNiztgV2izHvOYvJgM1dbhwZCmpC0NdqB6CIBMA9hxiGaTxM9V5APfj4a7bgknZnyee293jwi8Mj2NrtgcVoQKcaTF7f7oSBuOcQwzCNR9NZBADwlj19uGJNa8nnLl3lxXQ0hefOTqK31Q6DgQAo845XtzoWtedQLJnhamWGYepOUwrBbGzvUYrHDgwEtfiAZFOnGwcuBrX00oXygW+/hI/+z4FFeS+GYZj5wkJQwJYuN4yqFVAoBLft6MZgIIZnz04u+HOEEHjh3BTOcvCZYZg6UzMhIKI+InqMiI4R0REi+pB6vJWIHiaiU+rfLbVaw3ywmY3Y2OECAPS15AvBrdu74LWb8d0XLi74cwamYwgn0pgu0f6aYRhmKamlRZAG8FEhxFYAVwH4ABFtA/BxAI8IITYCeER9vKzYtkqpJ+grsAhsZiPeeFkPfnF4ZMEb+LFhpafRVDQJIRbH1cQwDDMfaiYEQohhIcRL6s9hAMcA9AC4A8A31NO+AeANtVrDfNm+SokTFLqGAOCte/uQzGTxwP7BBX3G8REl6JxMZ7XhNwzDMPVgSdJHiWgtgMsAPAegUwgxDChiQUQdZV7zPgDvA4DVq1cvxTI13rSnFyYjYWu3u+i5rd0e7Or14r9fuIj3XLMWRDSvz5AWAQBMzSThtDZlJi/DMMuAmgeLicgF4AcAPiyECM11vkQI8VUhxB4hxJ729va5X7CIeGxmvPvq8pv8W/euxvGRMA4OBEs+DwDBWApfevgkYmXu9o8Nh+BQC9oC0dIzkxmGYZaCmgoBEZmhiMC3hBD3q4dHiahbfb4bwFgt11ALbt/VDbvZiO/tKx80/t6LF3H3I6fw7ef7i56bURvb7VVbXExFOWDMMEz9qGXWEAG4F8AxIcQ/6p56EMCd6s93AvhRrdZQK2SLiv0XA2XPeUiddPa1p84hncnmPXdiNAwhgGsuaQMAzhxiGKau1NIiuBbA7wK4mYj2q39uA/A5AK8molMAXq0+XnFs7nTj1GgEmRLFZZORBF48P4UdPV4MTMc0UZDI+IAcijPFQsAwTB2pZdbQk0IIEkLsFELsVv/8TAgxKYR4lRBio/r3VK3WUEs2dbmRSGe15nR6Hjk2hqwA/u8bt2N1qwP3PHE27/njw2G4rSZs7fbAQMA0u4YYhqkjXFk8TzZ3KhlFJ0aKew89dHQEPT47dvR4cde1a/FSfwAv9U9rzx8bDmFLt1LB3OKwsEXAMExdYSGYJxs7XSACTo7mC8FMIo3fnJrAq7d1gojw5j19cNtMuPfJcwCU1hLHR8LY2q0UrbU4LWwRMAxTV1gI5onDYsLqVgdOFAjBE6fGkUxncculnQAAp9WEd75iDX52aBj/88JFDEzHEEmksaVLEYJWtggYhqkzXMW0ADZ1unGywDX00JFR+BzmvOlnf/KqDTg6HMLHfnAQN29R6udksZrPYS4ZZ2AYhlkq2CJYAJs73Tg3MaPNMU5lsnjk+BhetaUTJmPuV+uwmHDPu/fgtTu78ejxMRABm7sUIWh1VmYRvNQ/jdfc/QRmEunaXAzDME0LWwQLYFOXG2l1jvGWLg+eOj2BYCyF31bdQnosJgP+6W2Xodtjw9RMEg6L8quXMQIhxKztKvadn8axYWWWsowvMAzDLAYsBAtAnzm0pcuDBw8MwWMz4cbNpVtiGA2ET79uW96xVocFqYxAJJGG22Yu+1kTkQQAYDLC8QSGYRYXdg0tgHV+J0wGwsnRMOKpDH55eASv2d4Nq8lY8Xu0OC0AgOmZ2fsNTagCMDmTmP+CGYZhSsBCsAAsJgPWtztxYiSCR4+PYSaZwet3r6rqPVqdihUwV78htggYhqkVLAQLZFOnGydHw/jR/kG0u624an1bVa9vcUiLYPYNXloCbBEwDLPYsBAskM2dbvRPRfHY8XG8bme3Nu+4Ulqla2guiyCsuobYImAYZpFhIVggm9Q00GQmizt291T9ehkjkCmkiXQG77rnObxwPteCSQihswhYCBiGWVxYCBaIzBxa0+bArl5v1a93W00wGUizCI4OhfDk6Qk8cXJcOycUSyOVUbqcTkbq4xoKRJPIlui0yjDMyoeFYIH0tTrQ6bHirXv75jW2kojgc1gwpWYNHR5SWlQPB+PaOROqNWAyUF0sglA8hav/7lH8/PDIkn82wzC1h+sIFojRQHjiYzfDbJzf7GJAyRySweLD6vhLvRDIuMA6vxMjuuNLxVgojlgqg/OTM0v+2QzD1B62CBYBi8kw7yH2gJI5JNNHDw0qQjAUjGnPy9TRTV1uhBNpxFOl5yDXimAslfc3wzCNBQvBMqDVacH0TBLxVAYnR8MgAoYDcQiRHxeQ8Yil7lYqBSDA7bIZpiFhIVgGyH5DJ0fDSGcFLl/dglgqo23A45EkiICNHS4AS59CmhMCtggYphFhIVgGtDosmI6mcFCND7x6m9K0TsYJJiMJtDgs6PBYlcdLXFQWVAUgwK4hhmlIWAiWAS1OCzJZgWfOTMJrN2OvOstgWI0TTEQS8LssaHOqQrDkFoHS+jrIFgHDNCQsBMuAFofSb+jJ0xPY3uNBj88OABgKSIsgiTanFW0upfhsyS0C6RqKcYyAYRoRFoJlgKwuDsZS2N7jRbvbCpOB8i0CtxUuqwkWo6GmtQTHR0I4Mx7JO8YxAoZpbFgIlgGtauM5ANjR44XRQOj02DCcZxFYQERoc1nm7RoaDMTw/m/uQyhefkP/xP2H8JkHj+Qdk0KQSGeXPHWVYZjaw0KwDJCN5wBg+yqlTUWX14bhYBzxVAbhRBrtbiU+oAjB/FxDjxwbxc8Pj+CpUxNlzxkPJzAezn//kC5IzFYBwzQeLATLAOkacttMWNPmAAB0e20YDsY0N1Cbek6r0zrvOoKTo2EAyvzjcgSiKW0IjiQYS0E2VeU4AcM0HiwEywCnxQiL0YDtq7xahfIqnx3Dwbh2d+53KRaB32kp2qgr5eSI4vt/qT9Q8vlkOotIIo3pggZzwVgK3V4lgM0WAcM0HiwEywAiwvUb/bh1e5d2rNtrQyKdxSn1Ll5mDLU6LfPKGhJC4IT6XocGg0ims0XnyFhAJivy2kkEYynNUmEhYJjGg4VgmXDve/bizmvWao/lHfhhtfeQtAjaXFbEU1lEk+mq3n88nEAwlsJV61uRTGdxZChYdI6+hYR0SSXTWcRSGU0IgjrXkBACM4nq1sEwzPKDhWCZ0u21Acg1ocsJgVpLUKV7SFoDb9u7GkBp99C07m5fBqSlZbC61Qkg3yL45ZER7P2/v+JmdAyzwqmZEBDRfUQ0RkSHdcd2E9GzRLSfiF4koitr9fkrnW6fIgRHh0NwWoywW4wAckHjiSozh06MKEJw/UY/enz2kgFj/bhMGZCWm/wqnw1mI+W1mTgyFEI0mcFoaOlbYzMMs3jU0iL4OoBbC479A4DPCiF2A/gr9TFTAr/TCrOREE9l0aZaAwC0n6vNHDo5GlbaVLisuGy1Dy9fKBYCvWtookAIPHYzvHZLnkUgeyFx3IBhVjYVCQERfYiIPKRwLxG9RES3zPYaIcRvAEwVHgbgUX/2AhiqesVNgsFA6FLdQ35Xrs5AWgTVuoZOjkawsUNpY3356hYMBeNFQ270G/qU+v6yhsBrN8PnMOfVFMjK52luT80wK5pKLYK7hBAhALcAaAfwewA+N4/P+zCAzxPRRQBfAPCJcicS0ftU99GL4+Pj5U5raGTAON8iUF1DVWQOZbMCp0bD2NylCsGaFgDF9QTT0RTMRoLHZsLUTH6MwGs3w2c359URyMpnnlPAMCubSoVAjt+6DcDXhBAHdMeq4f0A/lQI0QfgTwHcW+5EIcRXhRB7hBB72tvb5/FRK59uzSLICYHDYoLdbNTu2CthMBDDTDKDTepgm23dHlhNBrxU4B4KRJPwOSzwu6xFriGfahFIq0EIUZVrKJJI4zMPHtGyoBaKEEq3Vm55wTALp1Ih2EdED0ERgl8SkRtAcSL63NwJ4H715+8B4GDxLEiLQO8aAtQ2E1XECE6NKYHiTZ3KYBuLyYAdPd4SFkESLQ4zWp0WTWjKxQiCsRRi6iY8PYcQTM0k8Y7/eBZff/o8Hnh5sOJ1z8YvDo/g7f/xLO576tyivJ+eI0NB3Hb3EwjP0pOJYRqJSoXgvQA+DmCvECIKwAzFPVQtQwBuVH++GcCpebxH07DKV2wRAEqcoFTW0Ie/+zLueeJs0fETakXxRtUiABT30OHBEFKZnJ4Hoin4HJa8orVgLAWnxQiz0QCfw6wJw7AuvjCba2goEMOb/+1pnBgJo8VhxrmJmTmvey5C8RT+P7Ux3oP7Fz/MtP9iAEeHQxiYjs19MsM0AJUKwdUATgghAkT0LgCfBjCrjU9E3wHwDIDNRDRARO8F8AcAvkhEBwD8LYD3zX/pjU8uRlBoERT3Gzo3MYMH9g/h8ZPF8ZSTo2F0eWzw2s3asfV+J5KZbF7qZyCags9uznv/YCylvc5nNyOSSCOVyWqBYvm6cvzBf76IsVAC//XeV+Cq9W2LIgSf/8UJTEQSeNMVvTg+EtaqryVCCJweC+OrvzmD/36hv+r3D6mDeLhYjmkWTBWe968AdhHRLgAfg+Lb/0/k7u6LEEK8vcxTV1S1wiZmV58Xu3q92N3nyzve5rTg6FAo79gPVZdLYedQQKkh2NTlzjsmM5KGg3H0tihVw9PRJHb3+dDmtGBqRuk3FIyl4JFCoA7QCcZSmkXQ12ovmzU0EozjyFAIn7ptK65c14pfnxjDw0dHkcpkYTbOL3N534VpfPO5C3jPNWvx/psuwf0vDeDHB4bwkVs2q89P4U//+wD6p6IAAIfFiLeqRXSVIl1CERYCpkmo9H9jWgghANwB4G4hxN0A3HO8hlkgHW4bfvTB67SNWtLb4sBYOK7dXQshNN97YXFXJitwejyCzWp8QLJKnYImN3QhhGIROM1oc1mQFcqM4mA0ZxFIQQhEUxgOxGE0EDZ1uMtaBM+cVdpdX31JGwBgnd+JdFbM2+UihMCnHziMLo8NH71lMzrcNlx9SRt+fHAYQgjEUxl89H8OIJMV+Js3bMdd165DNJmpuh1HOC4tAg5EM81BpUIQJqJPAPhdAD8lIiOUOAFTB97+ij5YTUZ84aETAJR2Ef1TUazzOzEdTeU1lOufiiKZzubFB4BcRtJwQNmUY6kMkpksWtQYAQBMzSTyXUMOOUktieFgHB1uZXxmudbUcgbztm6ldGR9u9Km4txEpOT5c/HyxQCODYfwp7+1CS6rYszevnMVzk3M4PBgCP/++Fmcn4zic7+zA++6ag22divXPBGuLr1VDu5h1xDTLFQqBG8FkIBSTzACoAfA52u2KmZWOtw2/P716/DTg8M4NBDEAy8PwmY24J2vUFwg47pA8qB6991XYFW4bWa4rCbNIpCZPz67WQtOT0SSRTECQLUIgjF0e21ocVgwHU1BMRjzeebsJF6xrhUGdZjBOr9ilZwdn1+c4IcvDcJqMuA1O3JdWm/d3gWzkfCVx07jK78+jdft7Mb1G5V0Y+06quzWKi0Cdg0xzUJFQqBu/t8C4CWi1wGICyH+s6YrY2blfTesR4vDjL/92TH85OAQXr2tS7vj1ruHhtSgrsxA0iOH3wDAtBoc9uVZBAVC4NALQRzdXjt8DovWoVTPwHQUF6dimlsIAFocZnjt88scSqaz+MnBIdxyaRfctpwx6nNYcMPGdvziyAgsRgP+8nXbtOc0ISgRN5kNGSOo1qXEMCuVSltMvAXA8wDeDOAtAJ4jojfVcmHM7LhtZnzglRvwzNlJTEdTeONlq9DhVjb7sVBu45PVvzI4rKdbHX4D5DJ/WhxmrY3FcDCOWCqjswiU49PRpGYR6MVBzzNnJgEAV63PCQERYZ3fOS8h+M3Jce06C3n9buXYR2/ZhE5P7jq1Tq1V9mWSWUMRjhEwTUKlWUOfglJDMAYARNQO4FcAvl+rhTFz866r1uBrT51HPJXB9Rvbteyd8XDOIhgOxuB3WWE1GYte3+2xadlH8rUtTos2OlP68r3qZu+2mUAEXJiMIp7KostrQ4v63HQ0qQWgAcUt1OIwY3NBbGJ9u1MTiWr44cuDaHVaNLePntt3rkKb04prdNYHoGvHMU+LgGMETLNQqRAYpAioTIJnGdQdm9mIe+7cg2gyA7PRgDanFQYCxnQb32AgVtItBCitriciCSTTWa29tM9uhtloyHPhSIvAYCB47WYcH1HEY5XPrgWQ9RaBEALPnpnEVevbtPiAZL3fiftfGkQ0mYbDUtk/v1A8hYePjeLte/tKpp0aDITrNvqLjltNRrhtpqotglzWEAsB0xxUupn/goh+SUTvIaL3APgpgJ/VbllMpWzt9uAKtYmc0UDwu6x5MQLFl19aCFapBWujoTgCuhgBoNxNy6CuR1eI5rObcXxYKeDqKuMa6p+KYigYz4sPSGTA+PxEtOJr/MWhESTTWbzx8t6KXyNpd1nzgudzkckKhBO1CxZ/89kL+KNv7Vv092WYhVBpsPjPAXwVwE4AuwB8VQjxF7VcGDM/Oj02zSIQQmA4ENMqlAuRcYOhQAzTUaWVhMWk/JNoc1q0+IG+ItnrsGgb5SqvHS2OXNxAIl0/V68vJQRKQPtsFSmkDx4Ywjq/E7t6vRW/RuJ3WbVpa5Wg3/xnahAsfuH8FH51dKxklhXD1ItKXUMQQvwAwA9quBZmEehwW7UNPBRPYyaZKesaksdHQnGt86ikzZnrb+QtsAgAxfpod1u1XkX6fkPPnZuC32XFho78IjYAWOtX0ljPVZFCemQoiFu3d4Oo+oa3bS4LTo1VLjr6eQu1CBaHYikkM1lMzSTz2oszTD2Z1SIgojARhUr8CRNRaLbXMvWhw2PFmBoslqmh5S0C5fhQII5ALKW5eQCgVdffKE8I1HM63VYYDQSb2Qi72ZjnGjo+EsaOHk/JjdthMaHba6s4c2hqJonpaAqXqKmx1eJ3Wasa6ynjA0YDIVoD11BIff/hII/3ZJYPswqBEMIthPCU+OMWQnhmey1THzrcNkzOJJHOZLXUUX02jx6X1QS3zYSRYExtQV08CQ0obRHo01FbHGatIE0IgQuTM1jrL79xr/M7cbZCITgzrtzNX1LCuqiENpfSOlvfZXU2ZMZQp9tak2CxtDh4zjOznODMnwajw2OFEEpV8GzFZJJVXjuGgnG1BXVuw5dCIFtQS7yqWHTrxMXnsCCotpkYCycQTWa0WEAp1vmdODseqchPfkZ162xon58Q+Kuc8Szv2Lt99poEi6XFMcJCwCwjWAgaDFlUNhqKYygQg9FA2rFSdPuU6uLpaLLANaRsoHprAMhZBN26wi2fziKQLp+1bbMLQSiennOgDQCcHovAajKUtWrmQg71qdQ9JC2CLo8NM8nMogd1ZR+j0WXqGjozHsGFyYW3CmdWFiwEDUaHW9nAx8IJDAfimi+/HN1eG4YCcQRjqTzXkF+1CDyFQqCKhd4iUPoNKXfc51UhmM0iqKb53JnxCNb5nbNew2zo+yZVgrxj7/LakMkKJNLzGcRXmlQmi2hSCUAvV4vg4z84qA39YZoHFoIGQ7ZYGAvHMRSM5W3Ypej22jE1k4QQyMsaksHiIotACoE33yIISotgcgYW4+x38Ktblcyhi1Nzt6M+Mz4z7/gAkBOCSlNIpQ9fXt9iuoci8dx7jYQ5D1bGAAAgAElEQVSqq3ZeKiYiSUxWMQ+baQxYCBoMv8sCImA0lMBwMD6nS6W7IOgrkemjhUKwo8eHq9e3YY9axAYoQhCIKR1Iz0/MoK/VPusdfLvqqio1REdPPJXBxenovOMDgK7NRKWuoUQaNrNBE8XFDBiHdDOQl6trKBRL5a2TaQ5YCBoMk9GANqcFY6G4IgRlqool+tRSfYxAikKhELS7rfjO+65Ch0cvIBZksgKheBrnJ6KzuoUAwGMzwWoyzFnxe35yBkLMP2MIUDKjrCZDxXe54XhKbdGt9GZazOE0spndKq9tWbqGhBAIxVN5tRRMc8BC0IC0u204PhJGMp0t215C0u3Tu3hyriGT0YAr1rRgRwXVvPJ10zNJnJ+cmTVQDChdSNvdVozNsRmeGVPiDfOtIZCf5a+izUQolobbZoJTHXyzmNXF8k57Q6cbwVgKseTy6m4aS2WQyiiCzpXPzQULQQPS6bFqXUXnjhHk39nr+cH7r8G7r1475+fJTKLjI2Ek0tlZawgk7e65N2dZQ7DeP3+LAFDcZZUGi0PxFDw2syYEixkjkBlJm1QLZ7lZBdJiyWSFFtRmmgMWggakw21FUi2gWlWmqljisJg0948+RlANLU7ldS9fnAYwe8aQfo2FMYJzEzP42lPntLvRM+MR9PjssFuKW2hXQ1sV/YZCcdUiUDujLmqMQN1oN6mtuUeWWZxAHxvgOEFzwULQgOjrBmYrJpN0e20gQt7kr2qQrqGX+wMAULFFMFYgBN99vh+f/fFRPK02rTszHllQfECiWASV1xF47GY4tRjB4ruGNnYq11SqujieyuDGzz+GR4+PLtrnVoo+NiBFi2kOWAgakE6PkvFjNRm0sZOzscpnh9dunneuvnQpHRoIwmoy5BWblaPdZUMgmkIinXNBDAaUdNJ/euQUslmBM2MzC4oPSJQOpMmK/N7heBoemwkuzTW0iMHieBpEueB3KdfQuYkZXJiM4sDF4KJ9bqWwRdC8sBA0IDI9U7nTn3tzv/XSLty+s3gEZKV4bMqmGUtlsKbNUTSMphQdnuJCr8FADAZSupf++OAQYqkMLllA6qikzWVFOisQrCAbJhRTsoZkjGAxG8+FYim4rCZ4bGa4raaSrqH+KWVOw+TM0tcZBPMsAhaCZoKFoAGRm2y5rqOFvGVvH/76Ddvn/Xkmo0ETg7kyhiTtaqGXPk4wFIjhth3d8Lss+OyPjwLAoghBrs3E7AHjZDqLRDoLj80Es9EAi8mAyCJnDXlU91un11bSNdQ/qQjBRHjpi7r07iC2CJoLFoIGRLaZ6K4gPrBYyDnHlQSKASVGAOSEIJnOYiycwIYOF37/+vVak7hLOhbHNQTMXVQms3pkrMRlNS1qjCCsBqIBpZdRKddQPS0CjhE0LywEDUi72wqLyVDx3fliIFNIKwkUAzmrRc5OGAnGIYQSr3jXVWvgc5jhtpk0y2Eh5NpMzH6XLTuPeuzKZu2wGCsqKPv0A4fw9OmJOc8LxVJa76ZOj61kdfEFKQR1aPMQiqdgUTvNsmuouah4QhmzcrCajPjhH12DNUspBGrAuFLxkS0spEUgA8W9PjtcVhP++o7tGJiOzWsqWdFnVdhmQrMIrDmLYK46glQmi28+2w+L0YhrNvjznhsO5o8JDcXT6FGttC6vkjWVyYq8IP1FVQiqGaazWIRiabQ6LQhym4mmgy2CBuXSVV4t82UpkDUIlbqGLGpGU6EQyN5It+9ahfffdMkirc0CA1UiBMqmL903zgpcQ7KRXCCWfwe/78IUrv67R3FqNKwdC8VyMYIujw3prMirb8hkBQamozAZCKF4GslF7HxaCcFYCh67CR67qaxrKBRP4c++d0BrMjgXBwcCmrhVw0NHRvDPj56q+nXleOr0BB47MVbRuZORBP7xoRPIZJunupqFgFkUOr02eGwmLXW1EtpduVqCIVUIuuZoiTEfjAZCq9M6Z7BYukOk+6YSIZDiUehKkb7+07p5ybJGAch1idXHCYaDMaQyApeuUob/VTpMZ7GQwWyPzVzWInj+7BS+v28Az5+fqug93//Nl/ClX52sei3feb4fX3joJA5cDFT92lJ86eGT+NzPjld07sNHR/FPj57GiZHw3Cc3CDUTAiK6j4jGiOhwwfE/JqITRHSEiP6hVp/PLC1/dOMG/OD911TlymnXVRcPTsfQ7rbCZl5YFXE5/C4LhgKxWWsJCi0Cl9WImTlaLcgNM1Bwhywfy9nE2axAOJHWsquk4OlTSGXG0GWrlc6ui+0eOjYcwi1feryswIRUofLYywvBqBrTqWRtqUwWQ8EYpuchaPL39oWHTlT92lJMziTRPxWtqJZkUl1vPdxz9aKWFsHXAdyqP0BErwRwB4CdQohLAXyhhp/PLCFehxkb1dYJlaJvMzEUjM17ClklrGlz4PGT47j2c4/ir39yFGfHi4fihAqyhpyWyi2CwhoFKQTyjj+STEOI3Ht3eXKT5CTSirhstQ/A4m9ED+wfxMnRSMlrB5QYgdduhsdW3jUkA9wTBVXh+y5M498eP5N/bkhJAAjHq89AGgzE4LWb8cSpCTyjVpovhIlIArFUpqLmg1MsBIuHEOI3AArtx/cD+JwQIqGeU5nTjmlIpEUghMBgIIbeGgrBF9+yG1988y5sW+XBfz1zAbd++Ql86eGTiKdyd/yy8tdtzcUI5goWywBzoRDIx9LlFS7ISGpzKZPj9K6hC1NRmI2E7T1Kx9fFzhx6/MQ4gPI1AopryDSrRSDXW7ihfu/Fi/iHXxzPi2vIu/pqA8+heArheBrvvW4dOj1WfOGhEwvqhppIZ7Tfv7S6ZoOFoPZsAnA9ET1HRI8T0d5yJxLR+4joRSJ6cXx8fAmXyCwV7WpzvGAshaFArKK+SPPFZTXhd67oxT137sVTH78Zr9nRhbsfOYXb7n5Cu0MOx1NwWUxaZbTTasRMYvaWzGEtWFxoESibiXT9aPEH1SIwGgidbitGgrnNpn8qit4Wh1YHspi1BKOhOI6rPu9Sd/vZrNDSWz02c9n00VF1slrhJjkaiiMrlDiHpFAEK2U4oPzO1vmd+JNXbcS+C9P49Yn57wF6Qb1QgRDkXEPlhfj/f+QUvvXchXmvabmx1EJgAtAC4CoAfw7gf6iMU1kI8VUhxB4hxJ729valXCOzRMiisuMjYcRTWfTU0CIo/Ny733YZ/vOuKzEcjOO+p84BUDZI/Yxmp9WErADiKeUuN5MVRZkk0iJIprN51oUUhuECIdA39uvy2tA/lRsU3z8ZRV+rAy6rCZYKh+mkM9mKsoseP5nbSEvdoc8k08gKRag8dlPZmQTSlVVY+SyD/vrxo0OB/GuvlKFgLoPsLXv6sLrVgS8+PH+rIE8IKshgmlIFuND9ped7+wbwpYdPIp1Z2syuWrHUQjAA4H6h8DyALAD/HK9hGhQpBPvVzJBaxghKccOmdly7wY9fnxiHEEKdTpZLuXUVzCT47I+P4N33PZf3HiHd3a4+YCx/Hg3FteltQM41BADXbvBj34VpraiufyqKNa0OZZiOs7IZCp/84SG8697n5jzv8ZPjWquNUhuzfn0em7nsTAJNCIosAlUIpnMbrbQOZpKZqjbMIS2V2Aaz0YA/vnkDDg+G8Mix+XmSJ3SWVf/kzCxnKkypv/eJWYLcwVgKE5Gk1il3pbPUQvAAgJsBgIg2AbAAmLskk2lIZLvsA3USAgC4cXM7BqZjODsxk9cCAoA2kyCq9ht6/twUTo3mB1rDurtrfZxA/ixrBeR5Hp1F8Ppdq5AVwE8PDiMYTSEYS2F1qwMA4HdbK3INPXt2Cs+fm5p1/nM6k8WTpybwys0dsJkNeeIl0buupFVUaDnEUxlMqwKnjxGkM1ltrQM6IZAWAVDdgJ+hQAxGA2n/Pt54WQ/WtDnw5UdOzssqkBZBj8+uBeTLIYTIuYbK/E4VYVd+Dz/aP1T1epYjtUwf/Q6AZwBsJqIBInovgPsArFdTSr8L4E7BM/GalkKLoLdl6YXgpk2K2/HXJ8bzmsIByJtSlskKnJ2YQSCWytuM9P5vvRBMR5Oaq2soGNe5hnJCs7HTjS1dbjx4YEjboFa3KULQ5px7hkIkkdZe98Sp8j70AwNBBGMp3Li5vaz/Xx7zqjEC5Vj+5i3FZk2bA+F4WnOFTUSSkL8SvWtIHy+oJk4wFIijy2PTKq5NRgM++Mr5WwWyaO/yNS1zCkE0mUFCdbWV+/2H4ykIAViMBvzyyEieS3ClUsusobcLIbqFEGYhRK8Q4l4hRFII8S4hxHYhxOVCiEdr9fnM8sdjU3zhw8E4HBajNiltKelrdWB9uxOPnxwvtgh0A+wHpqNIprNqLCDn5gjnuYaUO8ms2vJ6a7ecRBbT7sILh//csbsHL/cH8KTaq0haBG3qDIXZODES0n7WxwAKefzkOAwEXLfBXzYjKKgrppPuq8LzZMbQ9lVKVpPcKKW7yECFrqG41uepkhbgkqFArChe9MbLerC61YG7HzlVtVUwEUnAZjZga7cbE5HkrNaJzBjq9FgxOZNEtkR1sbyW1+3sRiSRxqPHV37yI1cWM3WDiLQMmR6ffVH6Cs2HmzZ14Nmzk5iaSRYFiwFlSpm+Qng6mtugQ/GU5nuXG0Q4rtQMbOlSKoSHg3GE4ynYzUZYTPn/5W7f1Q0AuPdJJWCdEwLLnMN0jg0rWUBXrm3FE6cmSm5agCIEu/t88Dks8NhMJe/OtRiBTW8RFAiBGviW6a0yhiGFYHOXBwPTihUQS2YwNZPEli639nuqlKFgrKhzrslowAdv3oBDg8GqN97JSBJ+lxVrWpX2J7OlkEq30KZONzJZUZQNBuTiP7du74LfZcWP9g9WtZ7lCAsBU1eke6ge8QHJTZvbkUxnEUmkywaLz+iKsPRB4XA8jZ4WZfOWQiD7Dq31O2FVLR4lI6m491NviwNXrGnBRCQBv8uiiY/fqaTWhme5ez02HILHZsLbruzD1EwSR4ZCRedMzyRxcCCAGzd1AFDu+GdzDSm9hkrHCOSGL1tgSB/6qPr3njUtGA8nEE9lNLfQZlUIKnUNZbICI8F4yX8Pb7ysBx6bCY+UEIJAtLz1NDGTRJvLqomsPlOrEJkxJOdKl3IPSXFodVpw+65uPHZivCqLZznCQsDUFdlmuqcO8QHJletaYTMr/xXcJWIEhRaBvsFcOJ5Cj0+Z+awJgSoULQ4zur02RQjiqbIzoV+/S5kOJzcqINcxdTb30PGRMLZ0e3CDGud4/GTxBvnyxWkIAVy1vlW7vpLBYnXTVyaoqa6hghjBaCgOq8mgjdqUAePxUBwGAnb3KRXRA9NRLW1WCkGh+Jwdj5S0diYiCaQyoqQQmI0GrPU7NatDcnAggMv/+mEcHiw93nMykoDfadHiL7PVEsjf92YpBCUCxvJ79jnMuGN3D5LpLH55ZKTse64EWAiYuiLnEixVDUEpbGYjrl7fBiA/q8dlyVkEp8ciWofVQotAulNyFkFuo+jy2jASjGmzkEtx245uGKhQCOQMBWUjevr0BK793KPaHWo2K3B8OIStXW74XVZcusqD35wsTsDb3x+AgYAdvV71+kxlLII0XFYTTEaDJliF542GEujy2tCmDiHSLIJQAn6XFWv9yvovTsW0FFC5oeotgvMTM7j5i4/joaOjReuQr+spU1zY22LPy0wCgMODIWQF8OzZ0qmck5Ek2lwWeO1m+BzmWQPGMkawSRWwUimkQdX68Not2NXrhdNixLHhYmtsJcFCwNSVdpfyH76WVcWVcKN6V10uWHx6LIIr1ijN4AqFwG0zwecwa8cDuo2i22vHUCCuNXQrRbvbii++ZRf+4Ib12jF/wQyFnx8ewWAghp8fGgYADEzHMJPMYGu3R1v/vv7pInfO/oEgNnW64VBFTQaLC+/Gg7GUFqy3mAywm40lg8WdbhtsZiM8NlMuWByOo9NjQ6/qItNbBBs7FetB/17yjr7Uxi1TTsuNWe1rcWBwOr954AXV1XOohEWgpIMmNGFd0+qYUwgsJgPWqKJcyiKQ37PXbgYRocVpKWo6uNJgIWDqSrsWLHbMcWZtuXV7N7b3eLRAKKAEKK0mAy5MzSAUT+OKNYp7RQaLU5ksYqkM3DYzvPacRaB3HXSrs4kD0fKuIQB442W9uHRV7rNz4zWVz3pBbfv8k4OKEBxV70C3qEJww6Z2ZLICT5/Oba5CCBy4GNCa2AGKxZPKiLzMJwCq6yongqVmEoyF4uhUu6b63bm23mOhBDrcVrS7lMl4F6cVi8DvssBhMcFpMeZZBLLmYH+JFtNDBXMpCultsSORzubVTch5B4cGioUgFEsjlRGaFbO6zam5hpLpLD75w0N5LqXJmSTanIr1YDJQyRhBMJaCw5IL/Lc4LHkJBCsRFgKmrly7oQ03b+nANjUAWS+6vDb85I+vLxqs47SatIK37T0e2MwGbaOPaCmhpjwh0N8xdnuVATSDgVhZ11ApWhy5GEEwmsKJ0TBaHGY8f34KY+E4jo+EQJRzvVy+ugUuqykvTnB+MopgLIVdvTohKJMaqh+jCaBoJoEQQrUIFIHyu6xajGAsHEeHxwaDgdDbYsfFqSiGgnHtrt5dULsg/fBHBkNIpPNz8AcDsbw4RSG96p36RV2cQG7sZydmiq5LVhX7dRbBYCCmTpa7gG8/148fH8gVhU3NJNHqtMBgILS5StdyBGIpbTQroAj+NFsEDDN/1rQ5cd979i7pNLVqcFqNODOuuB42dLjgs1s0109YVxtQKAQuqwlmowFd6maYyYqyrqFSWEwGeO1mTM4k8OKFKQgBfOSWzRAC+OXhERwbDmFdmxN2i1E7/8bN7XjoyKjWzkEK2K6+fIsAKPb/h9RYh3ZeQb1BKJZGPJXV5ii0u6yYCCeQymQxEUlqA4n6WhwYmI5hOBBDt3qux24qaREkM1kcLch0UsZ72sqmEvepSQUyTiCEQP9kFGvVQHBhwFiKjhSC1W0OZLICR4dCuPsRZQKa/H6VtSlCIF9Tqs1HIJovmi0Oy6xZSysBFgKGmQXZZsJlNaHLY8u7+8vNLyiwCGJJzd/erZu45pnFNVQKWUvw/LkpWIwGvPmKXmzocOEnB4dxfCSsxQckr93RjckZ5XxAcb04LEYtFRJA2dTQkDqmMrfWfNeQHEgjJ6u1uxWLQN4xy3YQfa12XFRjBNK9U2hdSD+8XKOeoUDp1FGJdCHKOEMgmkI4kcZrdyr1GAcHCoVAWZ/MwpIB+U/cfwjheAqbO904O5HLCJuaSWhuJEUIii2CUCwFn0MvBOZ5Dd9ZTrAQMMwsSEvlknYniAg+h1mb16ufaOZzKEIghEAwmtso9ELgrsI1BCi1BBORBJ4/P4WdvV7YzEa8dkc3nj8/hQuTUa1YS/LKzR2wm434iRpQ3n8xgO09Xq1VA4CyqaEhXbAYKLYIZDGZFAK/y4JwPK0VZ0mLoLfFgUA0hUgirSUAuAuK2CYiSaz3O9HtteHl/kIhmH1Akd1ihN9l1SwCGfjd1etDb4u9KE4gs36kEKxRLYejwyG8dW8fXrW1A/2TUaRUK2oqkkSr06q9plT6biCWhM9u0R77HBaE4umKGus9dnwMd/9q8WYxLxYsBAwzC05NCJTsF589FxjUN5Lz2pWOnZFEWvEhq0LQ6rTAYlT+m1XjGgIAv9uCgekYDg0EsXedEqh+7c5ura9PoUVgtxjxqq0d+OXhEcSSGRwdCmm5/ZJSFkFGG6NZECPQuY9kMVmXJgTKZimD1lIg+lpyQX8ZIygUlakZJZ1zd58vzyKIpzKYnEmWTR2VKHEIxSKQbaXXtDmxs9eLg4P5wiKzflrVmEun2waLyQCX1YSPvHoz1vmdSGcFLk5FEU9lMJPMaKLRrsZBCjOsAtFiiwCorI3G/S8P4t4nz8553lLDQsAws6BZBGoRlc9h1uoEwgXBYkDZDALRJHzqxkNEml+9mmAxALQ5rRgMxJDOClypCsGmTjc2qmvZ0l08GlS6h77+9HkkM9kiIZBWib6oLKK1oNZbBPkzCaQQyLoPKQSymlke72vN3c2XswgmIwm0Oq24bLUP/VNRzf0iU07LpY5K9LUEMmOor9WOnb0+XJzKn5E8OZNAi8MMkyrGBgPhPdesxWdffyna3VasVwX+3MSMVkOgjxEk08XV3YEC66lFPb+SgPF4OI7IHMOO6gELAcPMgkMNxm7QhMCCYDSlzS8AZLBY2QwCajtpfVaJdA/Nlj5aCnlnSgSthgEA3nXVGmzpcpcswrtpcwccFiP+5denAeQHioHSweKQZtmY8s7TzyQYDSXgc5hhMyu/D787JwQGUkQLKGMRqNaF3PwmI0qK5mWrlWvar7qH5kodlfSpmT/ZrMCFyRm0u61wWEzYqab+HtSng0aSWg2B5JO3bcXvXNELQHH5AcDZ8RJC4M4vnAMUqyWZzsLr0GcNye9+7jjBWDiBrFBmNCwnWAgYZhaka2iDziJIZrKIJjOzWAT5rgMpBN4SvYZmQ25gW7s8eW6bO69Zi198+IaSmTV2ixE3b+lAOJ6G32XFKm++m8WmNr7Tu2r0nUclhS4kWUwmkfUfp0bDaHdbtTiEz2GG02KEgaA1FHTbzEhnldqFRDqDcCINv8uC7auU+IV0Dw1qVcVzWwSpjMBoOI7+qagWAL5UFYJDAzn3kNJwzlLyfZT1WtDqtODsRERrOKcPFgP5IytlarA+RiBdQ5VZBIqohKtowrcUsBAwzCz0ttjhc5i1zUZrMxFLIaR2FDUbDZoQSFeOfqPo0t0ZV4Nf3ZCkW6hSXqdm0Ozu85UUC+UOPefuKJynrP9ZnjeqKyYDcptlOiu0+ACguML6Wh3o8tg0d4y+diF3122F3WLE1m43Xr44jWPDIfz742dgNxvR6c2/gy+kryWXOdQ/GdWqgL12M9b5nXmZQxO6quJyrPc7cWZ8Rms411okBDmLQPaZ8hakjwKYs6gsnsrdPFQ7x7nWsBAwzCy8++q1eOQjN8Ksbmo5F1Ayb36BtABkFo3edXDdBj/2rGnRfMmV0q3eGV+l9kGqlJs2d6DTY8WNm0vP+lb8/8WuIW9BjED/3Ggoji5PbkO1mY3atcs7f8lV69u04DaQc4mF4yktC0e6vS7ra8EL56dxx1eeQiiexn+8ew+sJuOs1ycHGJ0Zi2A4FEefrkfTjh5vXquJiXBCE9RyrPM7cXZ8Jrc2Z34cZFInBDJjzJfnGpI9qGYXAn01dKFFIISoa9xgeVbxMMwywWIy5N1R+nSN5/RCIDdRmcWijxFct9GP6zZWP5p7V68X3/79V1QtBDazEc98/FUwGEoXZRVmBMm7/vw6glwsIZ1RWjro7/wBJasmHE+jo+D4Z15/acHnKe8bjKW1oTDSoti7rhX/9ewF3LS5HV948y5t850NGUN49uwkhMilhAKKFfTggSGcGY+gr8WBUDw9t0XQ7sL39g2gfyoKk4G030OLwwwiYFzvGooVi6bLaoLJQHO6hsZ0QlDYAfYNX3kKN27uwEdevWnW96gVbBEwTBW0OHJBYX1raYfFCLORcEEdji4DiAuBiHDNBn/ZDX02ZnuNks6pcw3FZ48RfH/fALICWNuW335DBoz1sYNS5FsEssBLee3tO7vxow9ci/vu3FuRCACK0HV6rHhKHRyvF4Lbd62CxWjAN54+r7mh2maJEQDAejVg/OL5abQ4LZo7zWQ0oNWR32YiGC0WAqW+ZO7q4vFwboZzoWvo2EgYD7w8WDergIWAYarApwUG811DRASv3YzzEzN55y1HPDYTwnkWQQpEubbb8hwA+NWxMfzljw7j+o1+3LF7Vd77yFkSHZ7ZN3CvPZeyWpiZQ0TY1eerWux6Wxyaq0XvGmp3W/H63avwvRcHcFYdJiRdPeWQmUPHR0KapSLxq600JPqGgnqU6uLZLQK9ayiiEwKZidQ/Fc1rd7GUsBAwTBXos4PCBcPu9XfaviqLx5aSQosgGEvBbTXlbcbyLv6nB4fR1+LAP7/jci34K5HZOJ1zCIHeIpiIJGE2UtU1FYXIOIHdbNQESXLXtesQS2XwFTWFdrasIQBY3eqEgYCsyAmUxO+2FAWLjQYq6o1VSQfSsTIxAn0h2mN1mn/MQsAwVWAzG2E3G4uCxUD+5l9tFfFS4rblB4snIsmiDVDOJPDYTLjnzj15rhCJdOV0zOEa0mcgKb18rAueTy0zh1a3Oorea9sqD65e34an1Jbcc7mcLCaDZlUUCUFB47lANKXNIdCjn0dRjvGwMsDHaKA815D+dY+dYCFgmBWBbDxXKARys7SbjVrh1XLEYzMjmc4inlKKmk6PRbQ6CT1/dfs2fOOuK7Xq20K2dHvgtBjzXDOlsJkNMBlIyxoq3Gzng7QIVreV/uy7rlun/TxXjABQUkgBlHYN6dpMFBYLSiqxCMbDytwGl9VU0iLY3uPB8+em6lJjwELAMFXitZsxEUloQ2n0x4HlHR8A8gPBqUwWZyci2NBR3K7i7Veu1qp/S/FbWzuw7y9fXdJa0ENEWr+hCbXP0ELp1VkEpbh5SwfWtDm0vkJzIcWutSCesKbNgWgygxG1xUYwlspLDZb4nIpFMFuwdyycQIdHCkG+aw4A3rC7B+mswJOnikeO1hoWAoapkhaHRWuDnOcaUjOF5toY642+A+mFyShSGaH1L6oGIqrY8nGrba31bZ4XgswUKhwkJDEaCJ+5/VL84Q3rK3JDycyh1gKRkhPrZJFaIFreIpAV5+UYDyfQ7rKqrrliIXjllg64baa6uIdYCBimSnwOs9bszG0rjgu0LELqaC3RWwSnx8IAcrOFa/aZNrPmGporr78S+lod+MZdV+JNas+gUrxySwc+esvmit5PdpctLD7b1u2B0UDawBv9rAk9LbpsslJkswITkQTa3VbtdyGRaadtTgtu2NSOx06MI5td2jRSFgKGqcLD8QUAAA6eSURBVBKfw4JEWuk9XypYvOxdQ7pisVOjSorlJWXiAIuF22bCWDiBqK7N80K5cVP7osVi9q5txf+541K8cktH3nGb2YiNHS7NIlBmTRSv36erLynFdDSJdFagw20t6sYq03fdNjNu3tyB8XBC6+q6VLAQMEyV6Df6UsHi5S4E+rz+U2MR9PjsWnO9WuGx5WosFsM1tNgYDYR3X722pLDs7FXaVmSyAqF4uoxFMHu/IZk62u62KUKQyA8Wu60mGA2ktQV58vTSxglYCBimSvLSREsEi7325bfR6cmzCMYiNXcLAYpgytbLcxV4LTd29HgxNZPEMXUIz+yuodIWgSwm6/BY4baZ8wrK9AFov8uKvlY7DhUM2Kk1LAQMUyX6GEB+sHhlWAQyRhCIJnFmPJI307jWnwkUB2SXOzt6lZkO8i691Pc710wCKQQyWBzWDf0JFgy62dnjK5q9XGtqJgREdB8RjRHR4RLP/RkRCSKqvhMXw9QZffqg3iLQhGCZZw1ZTQaYjYSjwyEk09mSNQSLjV4w/SvMItjS5YbJQFpaZ2khUC2CMm0mcq4ha958BkBpZKdvW76j14uB6fxJa7WmlhbB1wHcWniQiPoAvBpAfw0/m2Fqhn6jd+k2uPV+F/78tzfjlku76rGsiiEieGxmvHh+GgDmlTpaLXrBXGkWgc1sxKZON54/PwWgtGvIbDTAbTWVjRGMhxNwWoxwWk2aKMrMoWKLQB2wM7h0VkHNhEAI8RsAUyWe+hKAjwFYXkM7GaZC5FwBOZRGYjAQPvDKDYtSOVtrPHazdpe6lBaB1WSA07J8q67LsaPHi6SaKVYuBqQUlZULFse1dt2Fc6NDsVSe6+zSRhKCUhDR6wEMCiEOVHDu+4joRSJ6cXx8fAlWxzCVIS0C9wIbp9UTWVTW7bVVPUt5Xp9nzwVDF9pnqB7s6PVqP5eLASltJsoHi2VzPL1FIIQosgi8djPWtjlwcGDpAsZLJgRE5ADwKQB/Vcn5QoivCiH2CCH2tLeXnrTEMPVAxghWtBCoG89SWANA7ne1EqylUuzUCUG5ynHfLP2GxtViMkDfjTWNWCqDVEYUicuOXh8ODy5dLcFSWgSXAFgH4AARnQfQC+AlIlreDlWGKcBqMsJhMS7JnXStkD77jSV6DNXy8xarmGyp2dzlhtlIcFry3YF6Whzm8kIQ0guBtAjSWgFaobjs7PFiMBDLa4FdS5ZMCIQQh4QQHUKItUKItQAGAFwuhBhZqjUwzGLhs5tXuEWgrH0pagiAnBCsVIvAajJic5d71slzLQ4LAiWyhmLJDMKJdAmLIKX1GSoUgu1LHCeoZfrodwA8A2AzEQ0Q0Xtr9VkMs9TsXdeKy/p89V7GvJEb86alEgJVeCodR7kcefuVq4umtOlpcVgQTqSRymTzjmvFZAUWQSSRzk08KxICDwDg8BLVE9TslkYI8fY5nl9bq89mmFpz99suq/cSFoQckLKhfWlcQ26bGTt6vLh89coVz3e+Ys2sz7c4ZaFeSrv7B4DxiNLCWh5zWUwgUrKGpBAUDjJy28xY3+7EwSWyCFaubcswzLx525V9uGJtS8ne+rXAaCD8+I+vW5LPqhf66uI8IdAsAiV91GAguCzKcJpgmRgBoMQJnj1bKgN/8eEWEwzThLhtZlw+y9AZpnrK9RsaDeWqiiUutc2EFiMoIcjbe7wYCcUxFo7XaskaLAQMwzCLQLkOpCdHw3DbTPC78ntUyWCx0UBwl+j+ulPtcXR4CdxD7BpiGIZZBGQweFCdXic5PBTCpas8eYV0bptZswg8NlPJIrsdPV58/ff24vI1tbfc2CJgGIZZBDo8NnR6rHkVwelMFseHQ7h0lTfvXLfONVSuQM1uMeKmzR15fZpqBQsBwzDMIrG7z4f9F3NCcHZiBol0VksHlbjVcZWBWYRgKWEhYBiGWSR297Xg/GRUayEt/fuzWgTLYMY1CwHDMMwisVstMtyvuoeODIVgNRmw3u/MO08KQYgtAoZhmMZiZ68XBgL290shCGJrtwemgv5EHpsZyUwW4+GENkO6nrAQMAzDLBJOqwmbOt3YfzEAIQSOqBlDhejbTLBFwDAM02Ds6vXhwEAA/VNRhONprYGcHpeubsBXZtDNUsJCwDAMs4jsXu1DIJrCTw8NA0AZiyB/EE29YSFgGIZZRGTA+NvP9cNkIGzqLG7sp29hXthwrh6wEDAMwywimzrdcFiMGJiOYUOHCzZz8YxmvRCwRcAwDNNgGA2EHWpcoLB+QKKvFi43A3kpYSFgGIZZZHarcxcKK4olbBEwDMM0OFeubQWQixcUos8aWg5CUP9KBoZhmAbj5i0d+PEHr8OO3tKuIZPRAIfFiGQ6C4elOIaw1LAQMAzDLDJEVFYEJC6rCVmLKNmCeqlhIWAYhqkDbpsJot6LUGEhYBiGqQNumxnLwBgAwELAMAxTF95/0yX1XoIGCwHDMEwd+O1Lu+q9BA1OH2UYhmlyWAgYhmGaHBYChmGYJoeFgGEYpslhIWAYhmlyWAgYhmGaHBYChmGYJoeFgGEYpskhIZZLt4vyENE4gAvzfLkfwMQiLmc5wtfYGPA1NgbL6RrXCCHa5zppRQjBQiCiF4UQe+q9jlrC19gY8DU2BivxGtk1xDAM0+SwEDAMwzQ5zSAEX633ApYAvsbGgK+xMVhx19jwMQKGYRhmdprBImAYhmFmgYWAYRimyWloISCiW4noBBGdJqKP13s9C4WI+ojoMSI6RkRHiOhD6vFWInqYiE6pf7fUe60LhYiMRPQyEf1EfbyOiJ5Tr/G/ichS7zUuBCLyEdH3iei4+n1e3WjfIxH9qfrv9DARfYeIbCv9eySi+4hojIgO646V/N5I4Z/U/ecgEV1ev5XPTsMKAREZAXwFwGsAbAPwdiLaVt9VLZg0gI8KIbYCuArAB9Rr+jiAR4QQGwE8oj5e6XwIwDHd478H8CX1GqcBvLcuq1o87gbwCyHEFgC7oFxrw3yPRNQD4E8A7BFCbAdgBPA2rPzv8esAbi04Vu57ew2Ajeqf9wH41yVaY9U0rBAAuBLAaSHEWSFEEsB3AdxR5zUtCCHEsBDiJfXnMJTNowfKdX1DPe0bAN5QnxUuDkTUC+C1AO5RHxOAmwF8Xz1lRV8jEXkA3ADgXgAQQiSFEAE02PcIZRSunYhMABwAhrHCv0chxG8ATBUcLve93QHgP4XCswB8RNS9NCutjkYWgh4AF3WPB9RjDQERrQVwGYDnAHQKIYYBRSwAdNRvZYvClwF8DEBWfdwGICCESKuPV/p3uR7AOICvqe6ve4jIiQb6HoUQgwC+AKAfigAEAexDY32PknLf24rZgxpZCKjEsYbIlSUiF4AfAPiwECJU7/UsJkT0OgBjQoh9+sMlTl3J36UJwOUA/lUIcRmAGaxgN1ApVD/5HQDWAVgFwAnFVVLISv4e52LF/LttZCEYANCne9wLYKhOa1k0iMgMRQS+JYS4Xz08Kk1O9e+xeq1vEbgWwOuJ6DwUd97NUCwEn+piAFb+dzkAYEAI8Zz6+PtQhKGRvsffAnBOCDEuhEgBuB/ANWis71FS7ntbMXtQIwvBCwA2qlkKFiiBqgfrvKYFofrK7wVwTAjxj7qnHgRwp/rznQB+tNRrWyyEEJ8QQvQKIdZC+c4eFUK8E8BjAN6knrbSr3EEwEUi2qweehWAo2ig7xGKS+gqInKo/27lNTbM96ij3Pf2IIB3q9lDVwEIShfSskMI0bB/ANwG4CSAMwA+Ve/1LML1XAfFtDwIYL/65zYoPvRHAJxS/26t91oX6XpvAvAT9ef1AJ4HcBrA9wBY672+BV7bbgAvqt/lAwBaGu17BPBZAMcBHAbwXwCsK/17BPAdKDGPFJQ7/veW+96guIa+ou4/h6BkUNX9Gkr94RYTDMMwTU4ju4YYhmGYCmAhYBiGaXJYCBiGYZocFgKGYZgmh4WAYRimyWEhYFY0RBRR/15LRO9Y5Pf+ZMHjp+f5PqT+/ZmCxx9UO1MKIvL/v/buH0SqK47i+PeQhZgihKjpbRTBQLRQ0RiwkLWxCDZWIlgYEsyCICKiKNGAYJcioEmpsQpaiRsxrMJCcEncXcVG8A9sY+GKCIKoOSnuHXyuOzsuWmRmzgeGmXfnzp15zfzefY93brN/u9RKSTtqyuVtSTuIeA9SCKJXLAHmVQhqQu1cXisEttfP8ze1rJT0E7BQ0tfAj7V9lHIH7v0Z/WdNrZS0EDgMrKWEKh7u9qjq+H9IIYhecRz4StJ4zcH/QNIJSWP1qPobAEkbVdZ0+I1ykw+Szkv6u2bn76ptxynJmeOSztS21uxDdeybkm5I2tYYe0Sv1hk4I0m2rwM/A9uBzbYPANi+bvveLPvSLrVyM3DJ9rTtR8Al3oxEjpi3gc5dIrrCfmCv7S0A9Q/9se3Vkj4ERiX9UfuuAT63fbdu77Q9LekjYEzS77b3S9pte+Us37WVcmfwF8Di+pmr9b1VwApKpswo8GUtIDuB08BlScdsH5xjX9qlVnZNmmV0l8wIolcNUnJexilR3Ysop1oArjWKAMCQpAngL0pI2FLmtgE4a/ul7QfAFWB1Y+wp2/9SIkCWABO2h4CHts8DhzqM3y61smvSLKO7ZEYQvUrA97aHX2uUNlJin5vbm4B1tp9KGgEWvMXY7TxrvH4JDLjmuNg+Up87/Xm3S62couQvNdtHOowV0VFmBNErngAfN7aHgW9rbDeSltXFX2b6BHhUi8ByyhKgLc9bn5/hKrCtXof4jLLa2LX3shdFu9TKYWBQ0qf1IvFgbYt4JykE0SsmgReSJiTtoSxzeQv4R2Wh8ZPMPgO+CAxImgSOUk4PtZwCJlsXixvO1e+bAP4E9rlES8+LpCFJU5Qj+0lJv9a3LgB3KAmdvwDfAdierr9xrD5+qG0R7yTpoxERfS4zgoiIPpdCEBHR51IIIiL6XApBRESfSyGIiOhzKQQREX0uhSAios/9B79fZO7Blbr1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1367aa5eac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title('loss curve')\n",
    "plt.xlabel('Iteration*100')\n",
    "plt.ylabel('loss')\n",
    "plt.plot(plot_losses)       # batch_size = 1, so there is a big jitter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. 评估模型\n",
    "注意在训练中使用了teacher forcing，即直接将标签输入解码器，但测试时输入只能根据上时刻的输出来得出，一般取最大值对应的索引来表示输入的词的one-hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1\n",
    "hidden_size = 128\n",
    "input_size = lang_dataset.input_lang_words\n",
    "output_size = lang_dataset.output_lang_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_encoder = EncoderRNN(vocab_size=input_size, hidden_size=hidden_size, n_layers=2)         # prefer using gpu\n",
    "eval_decoder = DecoderRNN(vocab_size=output_size, hidden_size=hidden_size, n_layers=2)\n",
    "eval_attn_decoder = AttnDecoderRNN(vocab_size=output_size, hidden_size=hidden_size, n_layers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load encoder and decoder's weight\n",
    "encoder_path = os.path.join('snapshot', 'encoder.pth')\n",
    "decoder_path = os.path.join('snapshot', 'decoder.pth')\n",
    "attndecoder_path = os.path.join('snapshot', 'attn_decoder.pth')\n",
    "eval_encoder.load_state_dict(torch.load(encoder_path))\n",
    "eval_decoder.load_state_dict(torch.load(decoder_path))\n",
    "eval_attn_decoder.load_state_dict(torch.load(attndecoder_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define evaluate function\n",
    "def evaluate(encoder, decoder, in_lang, use_attn, max_length=MAX_LENGTH):\n",
    "    \"\"\"\n",
    "        func: evaluate the seq2seq model\n",
    "        encoder: the RNN encoder\n",
    "        decoder: the RNN decoder\n",
    "        in_lang: input language data(Tensor) dims: (N,seq)\n",
    "        max_length: the maximum length of the sequence\n",
    "    \"\"\"\n",
    "    seq_length = in_lang.shape[1]                              # get the length of the input language sequence\n",
    "    eval_mode = lambda x:x.eval().cpu()\n",
    "    encoder, decoder = map(eval_mode, [encoder, decoder])      # change the model to evaluation model and use CPU\n",
    "    \n",
    "    # 1.encode the input tensor into context\n",
    "    encoder_outputs = torch.zeros([batch_size, max_length, hidden_size])   # for attention!!! (N,max_l,hidden)\n",
    "#     print(encoder_outputs.shape)\n",
    "    encoder_hidden = encoder.initHidden()                                       # (1, N, hidden)\n",
    "    for seq_idx in range(seq_length):\n",
    "        encoder_ouput, encoder_hidden = encoder(in_lang[:,seq_idx:seq_idx+1], encoder_hidden)\n",
    "        # => encoder_output:(N,seq,hidden),seq=1  encoder_hidden:(1,N,hidden)\n",
    "#         print(encoder_ouput.shape, encoder_hidden.shape)\n",
    "#         print(encoder_outputs.shape, encoder_ouput[:,:].shape)\n",
    "        encoder_outputs[:,seq_idx:seq_idx+1] = encoder_ouput[:,:]             # [N, 1, hidden]\n",
    "       \n",
    "    # 2.decode the context into sequence\n",
    "    decoder_input = torch.LongTensor([[SOS_token]])                             # (N,1)\n",
    "    decoder_hidden = encoder_hidden                                             # context (N,1,hidden)\n",
    "    decoded_words = []                                                          # store the generated words\n",
    "    decoder_attentions = torch.zeros([batch_size, max_length, max_length])      # store the attention\n",
    "    # 2.1 use attention to decode \n",
    "    if use_attn:\n",
    "        for seq_idx in range(max_length):\n",
    "            decoder_output, decoder_hidden, attention = decoder(decoder_input, decoder_hidden, encoder_outputs)\n",
    "            # => decoder_output:(N,1,vocab_size)  decoder_hidden:(1,N,hidden)  attention:(N,1,max_length)\n",
    "            decoder_attentions[:,seq_idx:seq_idx+1] = attention.data\n",
    "            topv, topi = decoder_output.data.topk(1)                           # select the maximum value along the last dim\n",
    "            # => topi: (N,1,1) \n",
    "            max_index = topi[0][0]    # (1)\n",
    "            if max_index == EOS_token:\n",
    "                decoded_words.append('<EOS>')\n",
    "                break\n",
    "            else:\n",
    "                decoded_words.append(lang_dataset.output_lang.index2word[max_index.item()])  # return the word\n",
    "                \n",
    "            # max_index with respect to the input in the next step\n",
    "            decoder_input = torch.LongTensor([[max_index]])                    # get (1,1)\n",
    "    else:\n",
    "        for seq_idx in range(max_length):\n",
    "            decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden)\n",
    "            # => decoder_output:(N,1,vocab_size)   decoder_hidden:(1,N,hidden)\n",
    "            topv, topi = decoder_output.data.topk(1)                            # (N,1,1)\n",
    "            max_index = topi[0][0]                                             # (1,)\n",
    "            if max_index == EOS_token:\n",
    "                decoded_words.append('<EOS>')\n",
    "                break\n",
    "            else:\n",
    "                decoded_words.append(lang_dataset.output_lang.index2word[max_index.item()])\n",
    "            \n",
    "            # put prediction to the next input\n",
    "            decoder_input = torch.LongTensor([[max_index]])\n",
    "    if use_attn:\n",
    "        return decoded_words, decoder_attentions[:, :seq_idx+1]\n",
    "    else:\n",
    "        return decoded_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "source:  je suis l elue .\n",
      "target:  i am the chosen one .\n",
      "translation:  you . <EOS>\n",
      "--------------------------------------------------\n",
      "source:  vous etes minces .\n",
      "target:  you re thin .\n",
      "translation:  they re tired . <EOS>\n",
      "--------------------------------------------------\n",
      "source:  il est le president du comite .\n",
      "target:  he s the chairman of the committee .\n",
      "translation:  you . <EOS>\n",
      "--------------------------------------------------\n",
      "source:  tu n es pas chanteuse .\n",
      "target:  you re no singer .\n",
      "translation:  they re nuts . <EOS>\n",
      "--------------------------------------------------\n",
      "source:  je me rejouis de vous contenter .\n",
      "target:  i m glad i make you happy .\n",
      "translation:  . <EOS>\n",
      "--------------------------------------------------\n",
      "source:  il est tres protecteur .\n",
      "target:  he s very protective .\n",
      "translation:  you . <EOS>\n",
      "--------------------------------------------------\n",
      "source:  nous ne sommes pas encore morts .\n",
      "target:  we re not dead yet .\n",
      "translation:  you . <EOS>\n",
      "--------------------------------------------------\n",
      "source:  vous etes avec des amis .\n",
      "target:  you re with friends .\n",
      "translation:  they re not too . <EOS>\n",
      "--------------------------------------------------\n",
      "source:  je ne suis pas mariee .\n",
      "target:  i m not married .\n",
      "translation:  you . <EOS>\n",
      "--------------------------------------------------\n",
      "source:  nous sommes tous la .\n",
      "target:  we re all here .\n",
      "translation:  you . <EOS>\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "def test(encoder,decoder,n=10, use_attn=True):\n",
    "    for i in range(10):\n",
    "        sample_index = random.choice(range(len(lang_dataset)))       # randomly pick a index\n",
    "        sample_pair = lang_dataset.pairs[sample_index]               # get the string pair w.r.t index\n",
    "        in_lang, out_lang = lang_dataset[sample_index]               # get the tensor of the pairs     \n",
    "        in_lang, out_lang = map(lambda x:x.unsqueeze(0), [in_lang, out_lang])   # get (N,seq)\n",
    "#         print(in_lang.shape, out_lang.shape)\n",
    "        if use_attn:\n",
    "            output_words, decoder_attentions = evaluate(encoder, decoder, in_lang, use_attn=True)\n",
    "        else:\n",
    "            output_words = evaluate(encoder, decoder, in_lang, use_attn=False)\n",
    "        output_sentence = ' '.join(output_words)\n",
    "        print('-'*50)\n",
    "        print('source: ', sample_pair[0])\n",
    "        print('target: ', sample_pair[1])\n",
    "        print('translation: ', output_sentence)\n",
    "test(eval_encoder, eval_attn_decoder)\n",
    "# test(eval_encoder, eval_decoder, use_attn=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "注意\n",
    "\n",
    "1. CrossEntropyLoss及NLLLoss的标签都是1维度的，否则会报错\n",
    "2. decoder_output.data.topk(top_num)默认是沿着最后一维，也可以使用dim=2来指定特定的维度"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
